{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Happy Customer Bank"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. 调整树的参数：再次调整弱分类器树目"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据说明:\n",
    "数据集共26个字段: 其中1-24列为输入特征，25-26列为输出特征。\n",
    "    1. ID - 唯一ID（不能用于预测）\n",
    "    2. Gender - 性别\n",
    "    3. City - 城市\n",
    "    4. Monthly_Income - 月收入（以卢比为单位）\n",
    "    5. DOB - 出生日期\n",
    "    6. Lead_Creation_Date - 潜在（贷款）创建日期\n",
    "    7. Loan_Amount_Applied - 贷款申请请求金额（印度卢比，INR）\n",
    "    8. Loan_Tenure_Applied - 贷款申请期限（单位为年）\n",
    "    9. Existing_EMI -现有贷款的EMI（EMI：电子货币机构许可证） \n",
    "    10. Employer_Name雇主名称\n",
    "    11. Salary_Account - 薪资帐户银行\n",
    "    12. Mobile_Verified - 是否移动验证（Y / N）\n",
    "    13. VAR5 - 连续型变量\n",
    "    14. VAR1-  类别型变量\n",
    "    15. Loan_Amount_Submitted - 提交的贷款金额（在看到资格后修改和选择）\n",
    "    16. Loan_Tenure_Submitted - 提交的贷款期限（单位为年，在看到资格后修改和选择）\n",
    "    17. Interest_Rate - 提交贷款金额的利率\n",
    "    18. Processing_Fee - 提交贷款的处理费（INR）\n",
    "    19. EMI_Loan_Submitted -提交的EMI贷款金额（INR）\n",
    "    20. Filled_Form - 后期报价后是否已填写申请表格\n",
    "    21. Device_Type - 进行申请的设备（浏览器/移动设备）\n",
    "    22. Var2 - 类别型变量\n",
    "    23. Source - 类别型变量\n",
    "    24. Var4 - 类别型变量\n",
    "\n",
    "输出：\n",
    "    25. LoggedIn - 是否login（只用于理解问题的变量，不能用于预测，测试集中没有）\n",
    "    26. Disbursed - 是否发放贷款（目标变量），1为发放贷款（目标客户）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train_save.csv')\n",
    "test = pd.read_csv('test_save.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((87020, 52), (37717, 51))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFpFJREFUeJzt3Xu0JWV95vHvw02Qi4C2LtNAGmNHJTreWoImEy8oAk5oZEGCyxEGmcFkAHESHTHJsuNdEoRI4iWsQLiMw0UGtEWUQQSdOIo0F8UGWXQwQg8EWkFEGGEafvPHfg/s7j6Xorv2OWzP97PWXqfqrbdq/7ar5VlVb9VbqSokSerDZnNdgCTpV4ehIknqjaEiSeqNoSJJ6o2hIknqjaEiSeqNoSJJ6o2hIknqjaEiSerNFnNdwGx7xjOeUYsWLZrrMiRpbFxzzTU/qaoFXfrOu1BZtGgRK1asmOsyJGlsJPlx175e/pIk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPXGUJEk9cZQkST1xlCRJPVm3j1Rv6le/p6z5roEPQld89eHzXUJ0pOCZyqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTejDRUkvyXJCuT/CDJOUm2TrJ7kquS3JLkvCRbtb5Paeur2vZFQ8d5X2u/Ockbh9r3bW2rkhw/yt8iSZrZyEIlyULgncCSqnohsDlwKHACcHJVLQbuBY5suxwJ3FtVzwVObv1Iskfb77eAfYFPJ9k8yebAp4D9gD2At7S+kqQ5MurLX1sA2yTZAngqcCfwOuCCtv1M4MC2vLSt07bvnSSt/dyqeqiqfgSsAvZsn1VVdWtVPQyc2/pKkubIyEKlqv4PcCJwG4MwuQ+4BvhZVa1t3VYDC9vyQuD2tu/a1v/pw+3r7TNVuyRpjozy8tdODM4cdgd+DdiWwaWq9dXELlNse6Ltk9VyVJIVSVasWbNmptIlSRtplJe/Xg/8qKrWVNX/Ay4EXgXs2C6HAewC3NGWVwO7ArTtTwPuGW5fb5+p2jdQVadW1ZKqWrJgwYI+fpskaRKjDJXbgL2SPLWNjewN3AhcARzc+hwOfLEtL2/rtO1fr6pq7Ye2u8N2BxYD3wWuBha3u8m2YjCYv3yEv0eSNIORvaSrqq5KcgFwLbAWuA44FfgycG6SD7e209oupwFnJ1nF4Azl0HaclUnOZxBIa4Gjq+oRgCTHAJcyuLPs9KpaOarfI0ma2Ujf/FhVy4Bl6zXfyuDOrfX7/hI4ZIrjfAT4yCTtlwCXbHqlkqQ++ES9JKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3hookqTeGiiSpN4aKJKk3M4ZKkm2TbNaWfzPJAUm2HH1pkqRx0+VM5ZvA1kkWApcDRwBnjLIoSdJ46hIqqaoHgYOAv62qNwN7jLYsSdI46hQqSV4JvBX4cmvbYnQlSZLGVZdQeRfwPuCiqlqZ5DnAFaMtS5I0jmY846iqbwDfSLJtW78VeOeoC5MkjZ8ud3+9MsmNwE1t/cVJPj3yyiRJY6fL5a+/Ad4I/BSgqr4H/N4oi5IkjadODz9W1e3rNT0yglokSWOuy11ctyd5FVBJtmIwnnLTaMuSJI2jLmcqfwQcDSwEVgMvaeuSJK2jy91fP2HwjIokSdPqcvfXmUl2HFrfKcnpoy1LkjSOulz++jdV9bOJlaq6F3jp6EqSJI2rLqGyWZKdJlaS7IzTtEiSJtElHD4B/O8kF7T1Q4CPjK4kSdK46jJQf1aSa4DXAgEOqqobR16ZJGnsdH3z4w+BC4EvAr9IsluXnZLsmOSCJD9MclOb8mXnJJcluaX93an1TZJTkqxK8v0kLxs6zuGt/y1JDh9qf3mSG9o+pyRJ958uSepbl7u/jgXuAi4DLmYw/f3FHY//SeCrVfV84MUMHpo8Hri8qhYzeOnX8a3vfsDi9jkK+Ez7/p2BZcBvA3sCy4bGeD7T+k7st2/HuiRJI9BlTOU44HlV9dMncuAkOzCYI+w/AFTVw8DDSZYCr2ndzgSuBN4LLAXOqqoCvtPOcp7d+l5WVfe0414G7JvkSmCHqvp2az8LOBD4yhOpU5LUny6Xv24H7tuIYz8HWAP8Y5LrkvxDmz7/WVV1J0D7+8zWf2H7rgmrW9t07asnaZckzZEuZyq3Alcm+TLw0ERjVZ3U4dgvA46tqquSfJLHL3VNZrLxkNqI9g0PnBzF4DIZu+3WaThIkrQRupyp3MZgPGUrYPuhz0xWA6ur6qq2fgGDkLmrXdai/b17qP+uQ/vvAtwxQ/suk7RvoKpOraolVbVkwYIFHUqXJG2MLrcUfwAgybZV9UDXA1fVvya5PcnzqupmYG/gxvY5HPh4+/vFtsty4Jgk5zIYlL+vqu5Mcinw0aHB+X2A91XVPUnuT7IXcBVwGPC3XeuTJPVvxlBJ8krgNGA7YLckLwbeUVX/ucPxjwU+16bMvxU4gsHZ0flJjmRwFnRI63sJsD+wCniw9aWFx4eAq1u/D04M2gN/DJwBbMNggN5BekmaQ13GVCbe/LgcBm9+TNLpzY9VdT2wZJJNe0/St5hiSv2qOh3YYBLLqloBvLBLLZKk0fPNj5Kk3vjmR0lSb3zzoySpN9OeqSTZHHhbVfnmR0nSjKY9U6mqRxhMnyJJ0oy6jKl8K8nfAecBjz2nUlXXjqwqSdJY6hIqr2p/PzjUVsDr+i9HkjTOZhpT2Qz4TFWdP0v1SJLG2ExjKo8Cx8xSLZKkMdflluLLkrw7ya7trY07txdnSZK0ji5jKm9vf4efTSkG70uRJOkxXWYp3n02CpEkjb8usxQfNll7VZ3VfzmSpHHW5fLXK4aWt2Yww/C1gKEiSVpHl8tfxw6vJ3kacPbIKpIkja1OU9+v50Fgcd+FSJLGX5cxlS8xuNsLBiG0B+DDkJKkDXQZUzlxaHkt8OOqWj2ieiRJY6xLqNwG3FlVvwRIsk2SRVX1LyOtTJI0drqMqXweeHRo/ZHWJknSOrqEyhZV9fDESlveanQlSZLGVZdQWZPkgImVJEuBn4yuJEnSuOoypvJHwOfai7pg8J76SZ+ylyTNb10efvxnYK8k2wGpqvtHX5YkaRzNePkryUeT7FhVv6iq+5PslOTDs1GcJGm8dBlT2a+qfjaxUlX3AvuPriRJ0rjqEiqbJ3nKxEqSbYCnTNNfkjRPdRmo/2/A5Un+kcF0LW8HzhxpVZKksdRloP6vknwfeH1r+lBVXTrasiRJ46jLmQrAdcCWDM5UrhtdOZKkcdbl7q8/AL4LHAz8AXBVkoNHXZgkafx0OVP5c+AVVXU3QJIFwNeAC0ZZmCRp/HS5+2uziUBpftpxP0nSPNPlTOWrSS4FzmnrfwhcMrqSJEnjqsvdX+9JchDwu0CAU6vqopFXJkkaO53u/qqqC4ELR1yLJGnMOTYiSeqNoSJJ6s2UoZLk8vb3hNkrR5I0zqYbU3l2klcDByQ5l8Eg/WOq6tqRViZJGjvTXf56P3A8sAtwEvCJoc+JXb8gyeZJrktycVvfPclVSW5Jcl6SrVr7U9r6qrZ90dAx3tfab07yxqH2fVvbqiTHd//ZkqRRmDJUquqCqtoP+Kuqeu16n9c9ge84DrhpaP0E4OSqWgzcCxzZ2o8E7q2q5wInt34k2QM4FPgtYF/g0y2oNgc+BewH7AG8pfWVJM2RGQfqq+pDSQ5IcmL7/LuuB0+yC/Am4B/aeoDX8fgUL2cCB7blpTw+pf4FwN6t/1Lg3Kp6qKp+BKwC9myfVVV1a1U9DJzb+kqS5kiXCSU/xuBs48b2Oa61dfE3wH8FHm3rTwd+VlVr2/pqYGFbXgjcDtC239f6P9a+3j5TtU/2G45KsiLJijVr1nQsXZL0RHW5pfhNwBuq6vSqOp3BJag3zbRTO6O5u6quGW6epGvNsO2Jtm/YWHVqVS2pqiULFiyYpmpJ0qbo+j6VHYF72vLTOu7zOwzuHNsf2BrYgcGZy45JtmhnI7sAd7T+q4FdgdVJtmjfc89Q+4ThfaZqlyTNgS5nKh8DrktyRpIzgWuAj860U1W9r6p2qapFDAbav15VbwWuYPBuFoDDgS+25eVtnbb961VVrf3QdnfY7sBiBu93uRpY3O4m26p9x/IOv0eSNCJdJpQ8J8mVwCsYXHJ6b1X96yZ853uBc5N8mMFbJE9r7acBZydZxeAM5dD2/SuTnM9gPGctcHRVPQKQ5BjgUmBz4PSqWrkJdUmSNlHXCSXvZBPOAqrqSuDKtnwrgzu31u/zS+CQKfb/CPCRSdovwWn4JelJw7m/JEm9MVQkSb2ZNlSSbJbkB7NVjCRpvE0bKlX1KPC9JLvNUj2SpDHWZaD+2cDKJN8FHphorKoDRlaVJGksdQmVD4y8CknSr4Quz6l8I8mvA4ur6mtJnsrguRBJktbRZULJ/8Rg1uC/b00LgS+MsihJ0njqckvx0Qzm8fo5QFXdAjxzlEVJksZTl1B5qL2vBIA22eOkswFLkua3LqHyjSR/BmyT5A3A54EvjbYsSdI46hIqxwNrgBuAdzCYa+svRlmUJGk8dbn769E25f1VDC573dympJckaR0zhkqSNwGfBf6ZwdT3uyd5R1V9ZdTFSZLGS5eHHz8BvLaqVgEk+Q3gy4ChIklaR5cxlbsnAqW5Fbh7RPVIksbYlGcqSQ5qiyuTXAKcz2BM5RAGr/KVJGkd013++v2h5buAV7flNcBOI6tIkjS2pgyVqjpiNguRJI2/Lnd/7Q4cCywa7u/U95Kk9XW5++sLwGkMnqJ/dLTlSJLGWZdQ+WVVnTLySiRJY69LqHwyyTLgfwIPTTRW1bUjq0qSNJa6hMqLgLcBr+Pxy1/V1iVJekyXUHkz8Jzh6e8lSZpMlyfqvwfsOOpCJEnjr8uZyrOAHya5mnXHVLylWJK0ji6hsmzkVUiSfiV0eZ/KN2ajEEnS+OvyRP39PP5O+q2ALYEHqmqHURYmSRo/Xc5Uth9eT3IgsOfIKpIkja0ud3+to6q+gM+oSJIm0eXy10FDq5sBS3j8cpgkSY/pcvfX8HtV1gL/AiwdSTWSpLHWZUzF96pIkjqZ7nXC759mv6qqD42gHknSGJvuTOWBSdq2BY4Eng4YKpKkdUz3OuFPTCwn2R44DjgCOBf4xFT7SZLmr2nHVJLsDPwJ8FbgTOBlVXXvbBQmSRo/Uz6nkuSvgauB+4EXVdVfPpFASbJrkiuS3JRkZZLjWvvOSS5Lckv7u1NrT5JTkqxK8v0kLxs61uGt/y1JDh9qf3mSG9o+pyTJRvxvIEnqyXQPP/4p8GvAXwB3JPl5+9yf5Ocdjr0W+NOqegGwF3B0kj2A44HLq2oxcHlbB9gPWNw+RwGfgcfOlpYBv83gSf5lE0HU+hw1tN++3X62JGkUpgyVqtqsqrapqu2raoehz/Zd5v2qqjsnXjlcVfcDNwELGTzjcmbrdiZwYFteCpxVA98BdkzybOCNwGVVdU87U7oM2Ldt26Gqvl1VBZw1dCxJ0hx4wtO0bIwki4CXAlcBz6qqO2EQPMAzW7eFwO1Du61ubdO1r56kfbLvPyrJiiQr1qxZs6k/R5I0hZGHSpLtgP8BvKuqprtsNtl4SG1E+4aNVadW1ZKqWrJgwYKZSpYkbaSRhkqSLRkEyueq6sLWfFe7dEX7e3drXw3sOrT7LsAdM7TvMkm7JGmOjCxU2p1YpwE3VdVJQ5uWAxN3cB0OfHGo/bB2F9hewH3t8tilwD5JdmoD9PsAl7Zt9yfZq33XYUPHkiTNgS4TSm6s3wHeBtyQ5PrW9mfAx4HzkxwJ3AYc0rZdAuwPrAIeZPCgJVV1T5IPMbi9GeCDVXVPW/5j4AxgG+Ar7SNJmiMjC5Wq+icmH/cA2HuS/gUcPcWxTgdOn6R9BfDCTShTktSjWbn7S5I0PxgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN4YKpKk3hgqkqTeGCqSpN6Mfagk2TfJzUlWJTl+ruuRpPlsi7kuYFMk2Rz4FPAGYDVwdZLlVXXj3FYmzY3bPviiuS5BT0K7vf+GWfuucT9T2RNYVVW3VtXDwLnA0jmuSZLmrXEPlYXA7UPrq1ubJGkOjPXlLyCTtNUGnZKjgKPa6i+S3DzSquaPZwA/mesingxy4uFzXYI25L/PCcsm+0/lE/LrXTuOe6isBnYdWt8FuGP9TlV1KnDqbBU1XyRZUVVL5roOaTL++5wb437562pgcZLdk2wFHAosn+OaJGneGuszlapam+QY4FJgc+D0qlo5x2VJ0rw11qECUFWXAJfMdR3zlJcU9WTmv885kKoNxrUlSdoo4z6mIkl6EjFUNKOZpsJJ8pQk57XtVyVZNPtVaj5KcnqSu5P8YIrtSXJK+7f5/SQvm+0a5xtDRdMamgpnP2AP4C1J9liv25HAvVX1XOBk4ITZrVLz2BnAvtNs3w9Y3D5HAZ+ZhZrmNUNFM+kyFc5S4My2fAGwd5JNftpKmklVfRO4Z5ouS4GzauA7wI5Jnj071c1Phopm0mUqnMf6VNVa4D7g6bNSnTQ9p3KaZYaKZtJlKpxO0+VIc8B/m7PMUNFMukyF81ifJFsAT2P6SxLSbOk0lZP6Y6hoJl2mwlkOTMyoeDDw9fIBKD05LAcOa3eB7QXcV1V3znVRv8rG/ol6jdZUU+Ek+SCwoqqWA6cBZydZxeAM5dC5q1jzSZJzgNcAz0iyGlgGbAlQVZ9lMNvG/sAq4EHgiLmpdP7wiXpJUm+8/CVJ6o2hIknqjaEiSeqNoSJJ6o2hIknqjaEidZTkkSTXJ1mZ5HtJ/iTJZm3bkiSnTLPva5JcPHvVbvD9f5nk3XP1/Zo/fE5F6u7/VtVLAJI8E/jvDGYPWFZVK4AVo/riJFu0edWkJzXPVKSNUFV3M5hK/Zj2tPZjZyJJXt3OaK5Pcl2S7dtuOyS5KMmNST47dJbzi4njJjk4yRlt+YwkJyW5AjhhquMmeU+Sq9v7Qj4wdKw/b+/B+RrwvNn430XyTEXaSFV1awuGZ6636d3A0VX1rSTbAb9s7XsyeCfNj4GvAgcxeFXAdH4TeH1VPZLkS+sfN8k+DN4VsieDyROXJ/k94AEGMxu8lMH/z68Frtm0XyzNzDMVadNMNgvut4CTkrwT2HHostV323tpHgHOAX63w/E/3/pPddx92uc6BsHxfAYh82+Bi6rqwar6ORvO1yaNhKEibaQkzwEeAe4ebq+qjwP/EdgG+E6S509sWu8QNUn71uv1eWCG4wb4WFW9pH2eW1WnTfF90sgZKtJGSLIA+Czwd+vPyJzkN6rqhqo6gcHg/USo7Nlme94M+EPgn1r7XUle0NrfPM13TnbcS4G3t8thJFnYbiL4JvDmJNu0sZff7+u3S9NxTEXqbpsk1zOYBXctcDZw0iT93pXktQzOYm4EvgK8Evg28HHgRQz+o39R6388cDGDNxT+ANhuiu/f4LhV9VCSFwDfbm9w/gXw76vq2iTnAdczGMP5X5vyw6WunKVYktQbL39JknpjqEiSemOoSJJ6Y6hIknpjqEiSemOoSJJ6Y6hIknpjqEiSevP/AQfaa5oYJqxDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.Disbursed);\n",
    "pyplot.xlabel('Disbursed');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "样本不均衡，使用交叉验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "默认参数，此时学习率为0.1，比较大，观察弱分类数目的大致范围\n",
    "（采用默认参数配置，看看模型是过拟合还是欠拟合）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Disbursed']\n",
    "train = train.drop([\"Disbursed\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 87020 entries, 0 to 87019\n",
      "Data columns (total 51 columns):\n",
      "Existing_EMI                     87020 non-null float64\n",
      "Loan_Amount_Applied              87020 non-null float64\n",
      "Loan_Tenure_Applied              87020 non-null float64\n",
      "Monthly_Income                   87020 non-null int64\n",
      "Var4                             87020 non-null int64\n",
      "Var5                             87020 non-null int64\n",
      "Age                              87020 non-null int64\n",
      "EMI_Loan_Submitted_Missing       87020 non-null int64\n",
      "Interest_Rate_Missing            87020 non-null int64\n",
      "Loan_Amount_Submitted_Missing    87020 non-null int64\n",
      "Loan_Tenure_Submitted_Missing    87020 non-null int64\n",
      "Processing_Fee_Missing           87020 non-null int64\n",
      "Device_Type_0                    87020 non-null int64\n",
      "Device_Type_1                    87020 non-null int64\n",
      "Filled_Form_0                    87020 non-null int64\n",
      "Filled_Form_1                    87020 non-null int64\n",
      "Gender_0                         87020 non-null int64\n",
      "Gender_1                         87020 non-null int64\n",
      "Var1_0                           87020 non-null int64\n",
      "Var1_1                           87020 non-null int64\n",
      "Var1_2                           87020 non-null int64\n",
      "Var1_3                           87020 non-null int64\n",
      "Var1_4                           87020 non-null int64\n",
      "Var1_5                           87020 non-null int64\n",
      "Var1_6                           87020 non-null int64\n",
      "Var1_7                           87020 non-null int64\n",
      "Var1_8                           87020 non-null int64\n",
      "Var1_9                           87020 non-null int64\n",
      "Var1_10                          87020 non-null int64\n",
      "Var1_11                          87020 non-null int64\n",
      "Var1_12                          87020 non-null int64\n",
      "Var1_13                          87020 non-null int64\n",
      "Var1_14                          87020 non-null int64\n",
      "Var1_15                          87020 non-null int64\n",
      "Var1_16                          87020 non-null int64\n",
      "Var1_17                          87020 non-null int64\n",
      "Var1_18                          87020 non-null int64\n",
      "Var2_0                           87020 non-null int64\n",
      "Var2_1                           87020 non-null int64\n",
      "Var2_2                           87020 non-null int64\n",
      "Var2_3                           87020 non-null int64\n",
      "Var2_4                           87020 non-null int64\n",
      "Var2_5                           87020 non-null int64\n",
      "Var2_6                           87020 non-null int64\n",
      "Mobile_Verified_0                87020 non-null int64\n",
      "Mobile_Verified_1                87020 non-null int64\n",
      "Source_0                         87020 non-null int64\n",
      "Source_1                         87020 non-null int64\n",
      "Source_2                         87020 non-null int64\n",
      "Source_3                         87020 non-null int64\n",
      "Source_4                         87020 non-null int64\n",
      "dtypes: float64(3), int64(48)\n",
      "memory usage: 33.9 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一轮参数调整得到的n_estimators最优值（93），其余参数继续默认值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        #xgb_param['num_class'] = 9\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='logloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print cvresult\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds4_2_3_699.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-logloss-mean']\n",
    "        test_stds = cvresult['test-logloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-logloss-mean']\n",
    "        train_stds = cvresult['train-logloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators4_2_3_699.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='logloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    test-logloss-mean  test-logloss-std  train-logloss-mean  train-logloss-std\n",
      "0            0.603579          0.000057            0.603569           0.000053\n",
      "1            0.530354          0.000115            0.530317           0.000074\n",
      "2            0.469315          0.000142            0.469242           0.000099\n",
      "3            0.417807          0.000106            0.417727           0.000073\n",
      "4            0.373941          0.000102            0.373847           0.000099\n",
      "5            0.336305          0.000115            0.336205           0.000083\n",
      "6            0.303783          0.000136            0.303647           0.000136\n",
      "7            0.275519          0.000095            0.275363           0.000132\n",
      "8            0.250917          0.000124            0.250724           0.000152\n",
      "9            0.229370          0.000127            0.229142           0.000171\n",
      "10           0.210454          0.000147            0.210214           0.000129\n",
      "11           0.193885          0.000108            0.193597           0.000148\n",
      "12           0.179189          0.000172            0.178861           0.000160\n",
      "13           0.166226          0.000169            0.165858           0.000166\n",
      "14           0.154772          0.000147            0.154380           0.000177\n",
      "15           0.144622          0.000169            0.144179           0.000178\n",
      "16           0.135616          0.000209            0.135133           0.000172\n",
      "17           0.127651          0.000234            0.127146           0.000159\n",
      "18           0.120606          0.000232            0.120044           0.000155\n",
      "19           0.114317          0.000255            0.113725           0.000152\n",
      "20           0.108775          0.000272            0.108151           0.000092\n",
      "21           0.103866          0.000283            0.103181           0.000138\n",
      "22           0.099504          0.000321            0.098743           0.000146\n",
      "23           0.095601          0.000366            0.094808           0.000152\n",
      "24           0.092149          0.000346            0.091295           0.000158\n",
      "25           0.089077          0.000404            0.088151           0.000143\n",
      "26           0.086338          0.000434            0.085359           0.000160\n",
      "27           0.083897          0.000433            0.082857           0.000139\n",
      "28           0.081771          0.000435            0.080673           0.000155\n",
      "29           0.079830          0.000422            0.078679           0.000159\n",
      "..                ...               ...                 ...                ...\n",
      "56           0.065086          0.000847            0.062259           0.000344\n",
      "57           0.064959          0.000869            0.062104           0.000331\n",
      "58           0.064921          0.000875            0.061981           0.000299\n",
      "59           0.064847          0.000871            0.061868           0.000296\n",
      "60           0.064800          0.000859            0.061743           0.000299\n",
      "61           0.064749          0.000903            0.061628           0.000286\n",
      "62           0.064684          0.000911            0.061532           0.000282\n",
      "63           0.064641          0.000950            0.061416           0.000267\n",
      "64           0.064614          0.000970            0.061331           0.000267\n",
      "65           0.064567          0.000980            0.061243           0.000267\n",
      "66           0.064513          0.001008            0.061164           0.000277\n",
      "67           0.064446          0.000959            0.061084           0.000292\n",
      "68           0.064415          0.000956            0.060991           0.000290\n",
      "69           0.064363          0.000962            0.060891           0.000281\n",
      "70           0.064297          0.000994            0.060815           0.000271\n",
      "71           0.064297          0.000988            0.060748           0.000264\n",
      "72           0.064270          0.001038            0.060683           0.000277\n",
      "73           0.064257          0.001030            0.060603           0.000282\n",
      "74           0.064218          0.001022            0.060525           0.000305\n",
      "75           0.064205          0.001041            0.060446           0.000291\n",
      "76           0.064200          0.001028            0.060381           0.000308\n",
      "77           0.064188          0.001025            0.060307           0.000307\n",
      "78           0.064156          0.001068            0.060237           0.000303\n",
      "79           0.064158          0.001092            0.060163           0.000305\n",
      "80           0.064148          0.001038            0.060092           0.000316\n",
      "81           0.064140          0.001067            0.059991           0.000303\n",
      "82           0.064120          0.001067            0.059933           0.000306\n",
      "83           0.064127          0.001060            0.059854           0.000325\n",
      "84           0.064095          0.001038            0.059768           0.000315\n",
      "85           0.064094          0.001061            0.059703           0.000320\n",
      "\n",
      "[86 rows x 4 columns]\n",
      "logloss of train :\n",
      "0.060452290900039236\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8XHW9//HXZ2aSJk3TNWlp0yXdKF2EgoGCoKCgUtSCiAiCP8Ht6pWLissFrw/vvXjxoiIuFy5XXMAFZFcQUVxYZBEkhVJoS0tJ0zZt2qbpknTJOp/fH+cknYbJ0pLJmWTez8fjPHKW75zzmTOTec9Z5hxzd0RERABiURcgIiLZQ6EgIiKdFAoiItJJoSAiIp0UCiIi0kmhICIinRQKIinM7Ktm9pOo6xCJikJhkDGzEWZWbWYfThlXbGYbzOy8lHEVZvagme00s11mttLMrjGzMeH0S8ys3cz2hF2VmX0mw7WfZmY1mVzGoUhXj7t/090/kaHlVZvZGZmYdyYM1Os12NbLUKdQGGTcfQ/wKeAHZlYajv42UOnu9wCY2VuAx4CngKPcfTRwJtAGHJMyu7+7+wh3HwGcB3zbzI4dmGcih8LMElHXIDnC3dUNwg64Ffg1cBpQD0xMmfYk8D+9PP4S4Mku4/4BfDhleAmwAthFEDJzU6bNDcftCtssSZl2FrASaAQ2AV8CioD9QBLYE3aTunleNwK/Dx//LDCzD+vjKODPwA5gNXD+4dQD/Afwq/Bx5YADlwIbgZ3Ap4HjgeXhc78hZTkzgUfC12M7cBswOpz2y3BZ+8NlfaUP67ga+NdwWc1AIhzeFD6X1cDpadbFicAWIJ4y7v3A8rD/BKASaAC2Atd3s05PA2q6mTYK+AVQB6wHvgbEwmlx4LvhOlgHXBaux0Q386oGzuhm2ieBteHr+kDHewYw4HvANmB3uI4WdPd6R/3/Opi6yAtQd5gvHIwBasN/vEtTxhcB7cBpvTz+ElJCIfyg2wUcGQ4fCewF3gnkAV8J/znzw+G1wFfD4XeE/4BzwsfWAm9NqfO4sL/bD5mUOm4NPwBOCD8EbwPu6OUxRQQf2peGjzkuXC/zD7Ue0ofC/wEFwLuAJuC3wHigLPxQOjVsPytcX8OAUuBvwPdT5n3Qh19P6zil/TJgClAIzAmf56SU+tIGJvAa8M6U4buBK8P+vwMfCftHACd2M49uXy+CQLgfKA7rWAN8PJz2aYIP5cnh+v4LhxEK4ftqe/h6DgP+B/hbOO3dwFJgNEFAzCX8YtTd662ub512Hw1S7r6T4BvmcOC+lEljCHYLbukYYWbfDo8r7DWzr6W0PTEcv4dgK+GXwKvhtA8Bv3f3P7t7K3AdwQfTWwi+iY4ArnX3Fnd/BHgQuDB8bCswz8xGuvtOd3/+EJ/efe7+D3dvIwiFhb20fy9Q7e63uHtbuLx7CXaJ9Uc933D3Jnf/E8GH+K/dfZu7bwKeAI4FcPe14fpqdvc64Hrg1B7m29M67vBDd9/o7vsJwn5Y+Fzy3L3a3V/rZt6/Jnw9zKyY4Nvzr1PWxywzK3H3Pe7+zKGsDDOLh7Vf5e6N7l5NsGXwkbDJ+cAP3L0mfJ9eeyjzT3ER8DN3f97dm4GrgJPMrDx8DsUEW4jm7qvcvTbl+b2R1zunKRQGKTO7mOAb2l+Ab6VM2kmwm2Jixwh3/4oHxxV+Q/BNusMz7j7ag2MKRwDzgW+G0yYR7BbomEeS4FtqWThtYziuw/pwGsAHCD6E1pvZ42Z20iE+vS0p/fsIAqgn04BFYcDtMrNdBB8oR/RTPVtT+venGR4BYGbjzewOM9tkZg3Ar4CSHubb0zrusDFl+lrg8wRbM9vCZU3qZt63A+ea2TDgXOB5d+9Y1scJtlJeMbPnzOy9PdSYTgnBFuL6lHGpr/+k1Lq79B+KrutnD8GuubLwi8gNBLsat5rZzWY2Mmz6Rl/vnKZQGITMbDzB/tRPAv8EnG9mbwNw970E++HPPZR5uvtWgm/X7wtHbSb4sO1YphHsxtgUTptiZqnvn6nhNNz9OXc/m2AXy2+BuzoWcyg1HYKNwONhwHV0I9z9MwNcz3+H8zza3UcCFxPs2ujQdXk9reO0j3H32939lPBxzsFfCFLbrST4QF0MfJggJDqmveruFxKsj28B95hZUd+fJtsJvo1PSxnX+foT7L6ZnDJtyiHMO1XX9VMEjOPA++yH7v5mgi8zRwJfDsd393pLHygUBqcbgN+6+6PhJvNXgB+H3woJhz9mZleGAYKZTQamdzdDMxtHcDByRTjqLuA9Zna6meUBXyQ42Pk0QejsBb5iZnlmdhpBmNxhZvlmdpGZjQp3iTQQ7PaA4Bv2ODMb1U/rocODwJFm9pGwnjwzO97M5g5wPcUEB5F3mVkZ4YdUiq3AjJThntbx65jZHDN7R/g6NxFspbSnaxu6HbgceBvBMYWO+VxsZqXhlsmucHS38zGzgtSOYEv0LuCa8HToacAVBFtGHc/rc2ZWZmajCQ6O9yavy3ISYf2XmtnC8Dl/E3jW3avD13dRuN72huujvZfXW/oi6oMa6g6tA84h+AY1usv4vwLXpAwvAh4i+KffBbwMXAOMC6dfQvDP0nHmzTaCfc7jU+bxfoIDhruBxwkP3IbT5ofjdodt3h+Ozwf+SLAbqwF4Djgl5XE/I9gFsIvuzz76r5Th0+jl4HTYbg7BGUt14fwfITgWcUj1kP5AcyKlfQ0pB/EJPgi/lrJOlobrcxnBh3xNStuzgQ3hsr7Uh3VczcEHpo8mOPbTSHAw/sF06zCl/VSCD/Dfdxn/q/D13kPwJeCcbh5/Wvj8u3azCI5d/Spc3xuBr3Pg7KMEwZZsPcHZR18g2LKwbpZTnWYZ/xVO+zTBQfOO5zs5HH86wRlHezhwpteI3l5vdb13Fq5gEZGMMLPFwP+5+7ReG0vktPtIRPqVmRWa2Vlmlgh3o/07wUkOMghoS0EGBTN7K/CHdNM8OHtKsoSZDSfYFXYUwXGP3wOfc/eGSAuTPlEoiIhIJ+0+EhGRToPuIlslJSVeXl4edRkiIoPK0qVLt7t7aW/tBl0olJeXU1lZGXUZIiKDipmt772Vdh+JiEgKhYKIiHRSKIiISCeFgoiIdFIoiIhIJ4WCiIh0ymgomNmZZrbazNaa2ZXdtDnfzFaa2Qozuz1dGxERGRgZC4Xwln03EtzkYx5woZnN69JmNsEt9k529/kEd5XKiKqV/+DZO7+dqdmLiAwJmdxSOAFY6+5V7t4C3EFwPflUnwRu9OA+rrj7tkwVs+35h1i06hp212/tvbGISI7KZCiUcfC9WWs4+N6zENxC70gze8rMnjGzMzNVzLDxswDYtmF1phYhIjLoZTIULM24rpdkTQCzCe7wdCHwk/D2fQfPyOxTZlZpZpV1dXWHVcyoSbMB2L351cN6vIhILshkKNRw8A27JxPcRrJrm/vdvdXd1wGrCULiIO5+s7tXuHtFaWmv13NKa8K0OQC0bn/tsB4vIpILMhkKzwGzzWy6meUDFwAPdGnzW+DtAGZWQrA7qSoTxRQVj6aeUcR39emaUCIiOSljoeDubcBlwMPAKuAud19hZleb2ZKw2cNAvZmtBB4Fvuzu9ZmqqS4xiaK9G3tvKCKSozJ66Wx3fwh4qMu4r6f0O3BF2GXcnuFllDW8OBCLEhEZlHLqF81tI6cx3rfT0twUdSkiIlkpp0IhXjKDuDlbN+oMJBGRdHIqFEZMDH6rsHOjfqsgIpJOToVCyZSjANi/Taelioikk1uhcMQUmjwP37Eu6lJERLJSToWCxeJsiR/BsMYNUZciIpKVcioUAHYPK2NU06aoyxARyUo5Fwr7i6dyRHstnkxGXYqISNbJuVCwMeUMt2bq67pehklERHIuFAomzARg+4ZXIq5ERCT75FwojJ4UXC21sXZtxJWIiGSfnAuFCdOOBKBte0YuxioiMqjlXCgUFBaxjbHEd+sS2iIiXeVcKABszy+jeJ8uoS0i0lVOhsLe4ZMZ11obdRkiIlknJ0OhfdQ0xrOD/Xv3RF2KiEhWyclQSJTMAGDbBl0tVUQkVU6GQnHHJbQ3rYm4EhGR7JKToVA6NbiEdpMuoS0icpCcDIUxJRPZ6wXYTl1CW0QkVU6GgsVi1CbKKGysjroUEZGskpOhALCraAbjm6qjLkNEJKvkbCi0jZ3NEWxnT+OuqEsREckaORsKwybOBWDzq8sjrkREJHvkbCiMm340ALs3vBRxJSIi2SNnQ2FS+VxaPU7bVt1XQUSkQ86GQiJ/GJvjkyjYrfsqiIh0yNlQAKgvnM64/dVRlyEikjUyGgpmdqaZrTaztWZ2ZZrpl5hZnZktC7tPZLKerlrGzKIsWUtz076BXKyISNbKWCiYWRy4EVgMzAMuNLN5aZre6e4Lw+4nmaonncSEo4ibs6nq5YFcrIhI1srklsIJwFp3r3L3FuAO4OwMLu+QjZn2JgB2VisUREQgs6FQBqTe3qwmHNfVB8xsuZndY2ZT0s3IzD5lZpVmVllXV9d/Bc46mqQbLVtW9ds8RUQGs0yGgqUZ512GfweUu/vRwF+An6ebkbvf7O4V7l5RWlrabwUWDB/Blth48na+2m/zFBEZzDIZCjVA6jf/ycDm1AbuXu/uzeHgj4E3Z7CetOoKpjF2r66WKiICmQ2F54DZZjbdzPKBC4AHUhuY2cSUwSXAgO/H2T9qNmXtm2hrbR3oRYuIZJ2MhYK7twGXAQ8TfNjf5e4rzOxqM1sSNrvczFaY2YvA5cAlmaqnO/HxcxhmrdSu1605RUQSmZy5uz8EPNRl3NdT+q8CrspkDb0ZOXU+LIft1S8xZdaCKEsREYlcTv+iGWDirIUANG1eGXElIiLRy/lQGDm6hDrGEK9fE3UpIiKRy/lQANg6bBqj91ZFXYaISOQUCsDekbOY1LoRTyajLkVEJFIKBYDSIxlh+9m6uTrqSkREIqVQAEZOCa6BtHXN0ogrERGJlkIBmDx3EQD7NjwfcSUiItFSKADFo8exySaQX6erpYpIblMohLYWzWHCPp2WKiK5TaEQailZwGTfQsOu+qhLERGJjEIhVDjtWAA2rvpHxJWIiERHoRAqOyo42Ny4TmcgiUjuUiiESiZOo57RxLa+FHUpIiKRUSik2FQwm5LGV6IuQ0QkMgqFFPvGzWNK+0aa9u+LuhQRkUgoFFLkTz6WPGtn42odVxCR3KRQSDHhyOMB2PFaZcSViIhEQ6GQYmL5XPZ4IV77YtSliIhEQqGQIhaPs2HYTEbv0sFmEclNCoUuGkfPZWprFe1tbVGXIiIy4BQKXcQmHcNwa6bmNf1eQURyj0Khi3GzgoPNda8+F3ElIiIDT6HQxZQjj6XFE7TWLIu6FBGRAadQ6CIvfxgbEtMYsXNl1KWIiAw4hUIaO0YvoLzpFdrb26MuRURkQCkU0ohNXUSx7We9ftksIjlGoZDGxAVvA2DbyicirkREZGBlNBTM7EwzW21ma83syh7anWdmbmYVmaynryZNn89ORhLbqDOQRCS3ZCwUzCwO3AgsBuYBF5rZvDTtioHLgWczVcuhsliM9cPnc0TD8qhLEREZUJncUjgBWOvuVe7eAtwBnJ2m3TeAbwNNGazlkDUdUcFU38SOutqoSxERGTCZDIUyYGPKcE04rpOZHQtMcfcHe5qRmX3KzCrNrLKurq7/K01j1Oy3ALD+xccHZHkiItkgk6FgacZ550SzGPA94Iu9zcjdb3b3CnevKC0t7ccSuzf9mLfS5jH2V/19QJYnIpINMhkKNcCUlOHJwOaU4WJgAfCYmVUDJwIPZMvB5oLhxVTnzWDk9heiLkVEZMBkMhSeA2ab2XQzywcuAB7omOjuu929xN3L3b0ceAZY4u5Zc4eb+jELmdH8Cq2tLVGXIiIyIDIWCu7eBlwGPAysAu5y9xVmdrWZLcnUcvtTYtqJDLdmqlfo1FQRyQ2J3hqY2Uygxt2bzew04GjgF+6+q7fHuvtDwENdxn29m7an9aXggTT56FOhEupf+RuzF54cdTkiIhnXly2Fe4F2M5sF/BSYDtye0aqyxIQps9nGWOKbsmaPlohIRvUlFJLhrqD3A9939y8AEzNbVpYwY9OIBUzaoxvuiEhu6EsotJrZhcBHgY7fE+RlrqTs0jyxgjLfSl3thqhLERHJuL6EwqXAScA17r7OzKYDv8psWdljzJzgWMKGFx+NuBIRkczrNRTcfaW7X+7uvzazMUCxu187ALVlhRlHn8I+H0brWv2yWUSGvl5DwcweM7ORZjYWeBG4xcyuz3xp2SEvv4C1hUdzxI6suV6fiEjG9GX30Sh3bwDOBW5x9zcDZ2S2rOyyf/IplCdr2LppXdSliIhkVF9CIWFmE4HzOXCgOaeUHv0uANZX/jHiSkREMqsvoXA1wa+SX3P358xsBvBqZsvKLuXzF7GTYqh6LOpSREQyqtdfNLv73cDdKcNVwAcyWVS2icXjrBtxHFN3V+LJJBbTXUxFZGjqy4HmyWb2GzPbZmZbzexeM5s8EMVlk7Zpb+MItrNh7ctRlyIikjF9+cp7C8HVTScR3CTnd+G4nFJ23LsBqF2m4woiMnT1JRRK3f0Wd28Lu1uBgbnTTRaZNH0+Wyghb/0TUZciIpIxfQmF7WZ2sZnFw+5ioD7ThWUbi8XYOPp4Zu59nvb29qjLERHJiL6EwscITkfdAtQC5xFc+iLnxGaeymj2UPXS01GXIiKSEX25zMUGd1/i7qXuPt7dzyH4IVvOKa84C4Dty/8ccSUiIplxuOdWXtGvVQwS4yZOY31sCkWbn4q6FBGRjDjcULB+rWIQ2TLuRGbvX87+vXuiLkVEpN8dbih4v1YxiBQtOItCa+GVZ34fdSkiIv2u21Aws0Yza0jTNRL8ZiEnHbloMXu9gOYVCgURGXq6vcyFuxcPZCGDRX5BISuKT2DGjidItieJxXXJCxEZOvSJdhiSsxcznh28ulwHnEVkaFEoHIZZJ7+fdjfql/426lJERPqVQuEwjCqZyJr8eYyvfSTqUkRE+pVC4TA1TD2DWe1VbF6fU7eWEJEhri+Xzk53FtLG8HLaMwaiyGxUtij4Uff6v98XcSUiIv2n15vsANcDm4HbCX60dgFwBLAa+BlwWqaKy2aTZx9DjU1k+Lo/Af8adTkiIv2iL7uPznT3H7l7o7s3uPvNwFnuficwpqcHmtmZZrbazNaa2ZVppn/azF4ys2Vm9qSZzTvM5zHwzNg8/lTmNi2jYfeOqKsREekXfQmFpJmdb2axsDs/ZVq3v2w2szhwI7AYmAdcmOZD/3Z3f5O7LwS+TbBVMmiMXHg2+dbGmqcfiLoUEZF+0ZdQuAj4CLAt7D4CXGxmhcBlPTzuBGCtu1e5ewtwB3B2agN3b0gZLGKQXT5jdsUZ7GIEvlKhICJDQ6/HFNy9CnhfN5Of7OGhZcDGlOEaYFHXRmb2WYKrruYD7+itnmwSz8vn1XHvYN72h9mzp4ERI0ZGXZKIyBvSl7OPJodnGm0zs61mdq+ZTe7DvNNdSfV1WwLufqO7zyQ4Wvu1bmr4lJlVmlllXV1dHxY9cIorLqDImln52J1RlyIi8ob1ZffRLcADBBfBKwN+F47rTQ0wJWV4MsFZTN25Azgn3QR3v9ndK9y9orQ0u24PfeTx76aOseStuDfqUkRE3rC+hEKpu9/i7m1hdyvQl0/m54DZZjbdzPIJTmU9aOe7mc1OGXwPMOh+CRZLJFh3xLuYv+8f7KzfFnU5IiJvSF9CYbuZXWxm8bC7GKjv7UHu3kZwIPphYBVwl7uvMLOrzWxJ2OwyM1thZssIjit89DCfR6TGnXgR+dbOK4/eFnUpIiJviLn3fMKPmU0FbgBOIjgm8DRwubtvyHx5r1dRUeGVlZVRLLpbnkyy6Rvz2JE3gaO/+njU5YiIvI6ZLXX3it7a9bql4O4b3H2Ju5e6+3h3Pwc4t1+qHCIsFmPT5PewoPlFtmyqjrocEZHDdrgXxLuiX6sYAia/9SPEzKl67JdRlyIictgONxTSnW6a08qOXEhVfAZj1/0u6lJERA7b4YbCoPrl8UB5OHYKR7WtZt3qF6MuRUTksHQbCt1cMrvBzBoJfrMgXXzo41+m1eNsfuRHUZciInJYug0Fdy9295FpumJ378slt3PO2COmsqL4Lczd+gBN+/dFXY6IyCHTndf6Wf6ijzGWRl78i36zICKDj0Khn809+WxqbTzDX9JZSCIy+CgU+pnF4myc/kHe1PKiDjiLyKCjUMiA2e/+dHDA+a864Cwig4tCIQPGTJjKiuKTmbtNB5xFZHBRKGRI5wHnP+uAs4gMHgqFDJl78hI2+zhGPH8jvV10UEQkWygUMsRicTbP/RjzWceKfzwSdTkiIn2iUMigBe/7Fxooounx70VdiohInygUMqigaBSrJ5/PcXufZN2a5VGXIyLSK4VChs163xW0Eaf2D9+NuhQRkV4pFDJszISpvFRyJsft+D1ba2uiLkdEpEcKhQEwafGXKbBW1jx4fdSliIj0SKEwACbOWsjyorewoOYuGht3R12OiEi3FAoDpOjtVzDGGll2n7YWRCR7KRQGyMyKd7Ky8DjmV/2UXTt3RF2OiEhaCoUBNPzM/2CsNbL83mujLkVEJC2FwgAqP+ZUlo84mYUbf0ndttqoyxEReR2FwgAb996rGcF+Xrn3mqhLERF5HYXCACs7qoLlo0/nzVvuYnNNddTliIgcRKEQgYnnXE0+rVTdd3XUpYiIHEShEIEJ0+ezvPR9LKr/LWteXhp1OSIinTIaCmZ2ppmtNrO1ZnZlmulXmNlKM1tuZn81s2mZrCebzLrgWppsGPvuv4JkezLqckREgAyGgpnFgRuBxcA84EIzm9el2QtAhbsfDdwDfDtT9WSbkSWTWLvg8yxsXcYzD90adTkiIkBmtxROANa6e5W7twB3AGenNnD3R9294ybGzwCTM1hP1jnmnCuojpczfek17G7Q5S9EJHqZDIUyYGPKcE04rjsfB/6QboKZfcrMKs2ssq6urh9LjFYskUdy8XeYyHaW3f7vUZcjIpLRULA049LerNjMLgYqgO+km+7uN7t7hbtXlJaW9mOJ0ZtR8S5eHPNOTqz9Fa++ohvxiEi0MhkKNcCUlOHJwOaujczsDODfgCXu3pzBerJW+QXX02oJ9t1zGa1tbVGXIyI5LJOh8Bww28ymm1k+cAHwQGoDMzsW+BFBIGzLYC1ZbdSEqVQfeyXHtL3Ik79Ou7EkIjIgMhYK7t4GXAY8DKwC7nL3FWZ2tZktCZt9BxgB3G1my8zsgW5mN+QtWPI5Vg2v4IS132Pt6pejLkdEcpS5p93Nn7UqKiq8srIy6jIyYldtFYkfvYXXErOYd+Vj5CUSUZckIkOEmS1194re2ukXzVlk9MQZVL/53zim7SWeuF2X1xaRgadQyDIL3nsZT7OQE1/7IS8vezbqckQkxygUso0ZC/75lzRZAcPv/zj1O3SXNhEZOAqFLDRy/FR2n3UT5ckaVv3kk7o2kogMGIVClpp+wnt4efZnOGXfX3jkju9GXY6I5AiFQhZ704Xf4JXhFZyy5lu88Ozfoi5HRHKAQiGLWTzBlE/8ij2xYib84VLWVVdFXZKIDHEKhSxXNHYibR+6nTE00vSLD1K/c2fUJYnIEKZQGASOOOokNp9xA3PaX+PV//swTc0tUZckIkOUQmGQmHnK+bxyzFWc2Pw0T970WZLJwfVLdBEZHBQKg8i893+FFXlv4oxdd/Hwj69isF2iRESyn0JhMDFj3pWPsWrcO1lcexN//Ol/KBhEpF8pFAYZiyc46jO388roU1lc833+8PNrFQwi0m8UCoOQJfKZ89m7WDPyJM5c9y0e+uV3FQwi0i8UCoOU5RUw67Lf8Frxm3lP1Td48Mdfp10Hn0XkDVIoDGKx/EJmfe5B1ow9jfdt/iEP//CzNLXodp4icvgUCoOc5RVy5GX3sXrS+zlr1208+f2LadjXFHVZIjJIKRSGglicOZ+8hdWzP8kZ+/7Aqm+9naoNNVFXJSKDkEJhqDBjzkXXUXXiNznO1hD/6ek888xTUVclIoOMQmGImXHmZ9l1/n2MjDUx/w8f4MG7fqwD0CLSZwqFIah03qkUfPYJdhZO5b0rv8TD132Uzdt1IT0R6Z1CYYgqLJnKlC8+zpryizhr3/003nAqf3vqiajLEpEsp1AYwiyvkCMv+V+2vvcXTLBdHP+nc7nvf/+N+oZ9UZcmIllKoZADJlScTdHnn6V27PGcu+0GNl3/Nv7y2KP6FbSIvI5CIUfkjZrIjMt/T+3p/8M028qpj36A+7/3z6yt2RZ1aSKSRRQKucSMiW/9f4z44gtsmLSYcxpup+jHi7jnZ9dR37g/6upEJAsoFHJQfEQJM//pNnZf8ADJovGct+Eb1F53MvfddweNTa1RlyciEcpoKJjZmWa22szWmtmVaaa/zcyeN7M2Mzsvk7XI64066lTKvvR3tp7+fSYldnPu8n9i1bWnct9v7lY4iOSojIWCmcWBG4HFwDzgQjOb16XZBuAS4PZM1SG9iMWY8NZLGXvly2xa9HXmxDZz7oufYMW1b+fOO3/B1t3arSSSSzK5pXACsNbdq9y9BbgDODu1gbtXu/tyIJnBOqQv8gopW/xFRl25ks2LvsbceA0fWvUv7Lj+BG770bW8tF4HpEVyQSZDoQzYmDJcE447ZGb2KTOrNLPKurq6filOupE/nEmLv8yoq1ZTf/r1jCuMc1Htf3PEzyq489pP8JtHnqJBu5ZEhqxEBudtacYd1onx7n4zcDNARUWFTq4fCIlhjHvrx+GUj7F31cM0PXoT59XdS/xvd/PU40fz2qT3MfXkD3Ly3GnkxXW+gshQkclQqAGmpAxPBjZncHmSCWYUzTuTonln4rtrqH38p8x76TZOrr2GfXdfx5/teLaVL2H6CWdx4pGTGJaIR12xiLwBlqlftZpZAlgDnA5sAp4DPuzuK9K0vRV40N3v6W2+FRUVXllZ2c/VyiFJJmld9xTbnvolo6sfoijZSIMX8gTHUTvpnZQuPIuT5k1jfHFB1JWKSMjMlrp7Ra/tMnk6Eza/AAAMPElEQVSpAzM7C/g+EAd+5u7XmNnVQKW7P2BmxwO/AcYATcAWd5/f0zwVClmmrZmWVx9l+3P3MGr9nyhq302Lx6lMzmHViEUw63Rmzj+eiukljBiWyQ1TEelJVoRCJigUslh7G8n1T1O/7PfYa3+hZO9aAOq9mGd9HhuKj8PKT6bsyGM5dto4Jo0qwCzdoScR6W8KBYlew2aaV/+Z3SsfoXDT0xS3BKe1Nnghy5KzWJM3h70lCymY9maml89gftkoBYVIhigUJLu4w44q2qqfYferT2ObnmN046vEwp+obPExvJwspyo2jcaRs/Hx8ygum0v5hDHMLC1i6tgi8hM6y0nkcCkUJPs174EtL9Gy8Xkaq/5BfNvLFO9ZR5x2ANrd2ODjec0nsc4nsbNgCi2jyomPm8nICVMpG1tM2ZhCykYXMr54GAmdGivSLYWCDE5tLVD/KmxdSfOWVeyvXY3Vr6Foz3oSyebOZi0eZ7OXUOMl1HgptZSyp+AImkeUERtZRv6YSYweNYrxxQWUFOczrmgY40YEfwvzddqs5J6+hoJOB5HsksiHCfNhwnyGHQ3DOsYnk9BYCzuqYMdrWH0147ZVMXZnNW9ufInClnpoA3aF3QZo8OFs9THU+SiqGU2lj2K7j6IhPpKW/HG0FY7DC8diw0vIKxrFqOH5FBfkUVyQYGT4d0TYP2JYgqJhCYqGxSnMi+u4hwxZCgUZHGIxGFUWdNPfSh6Qlzq9tQkaNsHujdBQC421DN9dS9nOGibu2UZsbw15+18grz28wF8b0Bh2QBsxdnkxu304uxjBLh/BborY7MNpZDgNPpy9FLLHC9lrBbTFh5PMG04yMRzyiyCvCM8fTl5eAQX5cYYl4hTkxRiWiDMsESM/tYsHf/PiQReMM/ITMRKxjvFGXjxGouNv7MBw0MaIx1I6C/4qrOSNUijI0JBXAONmBl0oQZo3eMte2Lsd9m2HvfWwrx727yCxr56SffWM3beL9n07Se7bgTVtwJobSLQ0YOmu2dgeds2po2I0UUCTDWO/57OffJo8n/2ex37Pp4mga/Y8mshnD3m0kKDZ84O/5NFM0N/iwbQ24rSSCMclaAkf0zGu1TvaBO3aSRDPyycRixELQyNmRjwGiVgMM8Jh6+yPdf41EnE7MB0wA7OgP2ZGLBb8hdTxB/otnF/HvM3AsIPmQzhfwnkagB3o73ycddSYMo9wGR3zMML6U55Pd9HYsczUFh3zxA48385pXebUNXNT109PupvcXaWdNXV57CmzSpk3aWSPy3qjFAqSW/KLgm7MtLSTY6S5SqQ7tOwJDow3N0JLY9Dfui8ImZa9Yf8e4i37KGrdR1HrfmjdH4xva4LWJrx1H966F2/bAW1NWGsTtDdj7S1Ysv8vMpgkTjtxkh4P+tsTtCfDfgv+Ji1Ge0c7i9NOMNxG0J8kRpI4bZ39Qdd1uD3sUvvbPUbSLPiLhcsJH+NGEiNpRtKDc9DaPU4Sgnm40djcjmPk5+expyUZ9OclaPcYe1shiZGXSLCvLUmSGPFYgnagpT2Yh8WCV7K1PViWxWK0JensT9IxHNSXxHCsc7idGGAHaiUWTrfOv8kgFjqnd51Hx3D6S8EduvJxG3jsy2/vl3l1R6Eg0hszGFYcdEw8/NnQw0dDsj0Ij7ZmaG8J+1sg2RoMt4d/O6c3B+OSrUF/su1Am2QrtLcRS7YRC/tJdnStwfGZg4bbwZPh/DrGtwd/vR2SLQfaeHvY335gXEf7g8alaX+oOj6dkin9HefFpO47PGg/IsH1E9LNJ0KOQSwOFjvQEW5OpY6zGFgcf914A2IkT/tqxmvNgtUlIsTiB7ZihqpkGBKePNAl2wEPg8PD6X4gUDr7uzwmdT6dAdTD4/AubXpahh8IM7ybtt5lvsku82kPAizst661HjTfro9LprQ/uPbYyPEZf5kUCiIyMGJpd85JltErJCIinRQKIiLSSaEgIiKdFAoiItJJoSAiIp0UCiIi0kmhICIinRQKIiLSadDdT8HM6oD1h/nwEmB7P5Yz1Gj99Ezrp3taNz3LhvUzzd1Le2s06ELhjTCzyr7cZCJXaf30TOune1o3PRtM60e7j0REpJNCQUREOuVaKNwcdQFZTuunZ1o/3dO66dmgWT85dUxBRER6lmtbCiIi0gOFgoiIdMqZUDCzM81stZmtNbMro64nSmY2xcweNbNVZrbCzD4Xjh9rZn82s1fDv2OirjVKZhY3sxfM7MFweLqZPRuunzvNLD/qGqNiZqPN7B4zeyV8H52k90/AzL4Q/l+9bGa/NrOCwfTeyYlQMLM4cCOwGJgHXGhm86KtKlJtwBfdfS5wIvDZcH1cCfzV3WcDfw2Hc9nngFUpw98Cvheun53AxyOpKjv8APijux8FHEOwnnL+/WNmZcDlQIW7LyC4Y/QFDKL3Tk6EAnACsNbdq9y9BbgDODvimiLj7rXu/nzY30jwD11GsE5+Hjb7OXBONBVGz8wmA+8BfhIOG/AO4J6wSc6uHzMbCbwN+CmAu7e4+y70/umQAArNLAEMB2oZRO+dXAmFMmBjynBNOC7nmVk5cCzwLDDB3WshCA4g83cJz17fB74CJMPhccAud28Lh3P5PTQDqANuCXev/cTMitD7B3ffBFwHbCAIg93AUgbReydXQsHSjMv5c3HNbARwL/B5d2+Iup5sYWbvBba5+9LU0Wma5up7KAEcB9zk7scCe8nBXUXphMdRzgamA5OAIoLd1l1l7XsnV0KhBpiSMjwZ2BxRLVnBzPIIAuE2d78vHL3VzCaG0ycC26KqL2InA0vMrJpgV+M7CLYcRoe7BCC330M1QI27PxsO30MQEnr/wBnAOnevc/dW4D7gLQyi906uhMJzwOzwDIB8ggM/D0RcU2TC/eM/BVa5+/Upkx4APhr2fxS4f6BrywbufpW7T3b3coL3yiPufhHwKHBe2CyX188WYKOZzQlHnQ6sRO8fCHYbnWhmw8P/s451M2jeOznzi2YzO4vg214c+Jm7XxNxSZExs1OAJ4CXOLDP/KsExxXuAqYSvLk/6O47IikyS5jZacCX3P29ZjaDYMthLPACcLG7N0dZX1TMbCHBQfh8oAq4lOBLZs6/f8zsP4EPEZzl9wLwCYJjCIPivZMzoSAiIr3Lld1HIiLSBwoFERHppFAQEZFOCgUREemkUBARkU4KBRER6aRQEOkDM1sY/talY3hJf12C3cw+b2bD+2NeIm+Ufqcg0gdmdgnB5ZAvy8C8q8N5bz+Ex8Tdvb2/axHRloIMKWZWHt705cfhjU7+ZGaF3bSdaWZ/NLOlZvaEmR0Vjv9geIOUF83sb+GlUa4GPmRmy8zsQ2Z2iZndELa/1cxuCm9cVGVmp5rZz8I6bk1Z3k1mVhnW9Z/huMsJLpz2qJk9Go670MxeCmv4Vsrj95jZ1Wb2LHCSmV1rZivNbLmZXZeZNSo5x93VqRsyHVBOcHmBheHwXQSXFEjX9q/A7LB/EcE1jiC4/EdZ2D86/HsJcEPKYzuHgVsJLmFgBFfIbADeRPCla2lKLWPDv3HgMeDocLgaKAn7JxFcIqKU4GqkjwDnhNMcOL9jXsBqDmztj4563asbGp22FGQoWufuy8L+pQRBcZDwsuFvAe42s2XAj4CJ4eSngFvN7JMEH+B98Tt3d4JA2eruL7l7EliRsvzzzex5gmvfzCe4C2BXxwOPeXCVzTbgNoIb2gC0E1zZFoLgaQJ+YmbnAvv6WKdIjxK9NxEZdFIvNNYOpNt9FCO48cnCrhPc/dNmtojgzmvLwou/9XWZyS7LTwIJM5sOfAk43t13hruVCtLMJ919Gzo0eXgcwd3bzOwEgqtwXgBcRnCJb5E3RFsKkpM8uKnQOjP7IASXEzezY8L+me7+rLt/HdhOcC+ORqD4DSxyJMHNaHab2QQOvvFK6ryfBU41s5Lw3uIXAo93nVm4pTPK3R8CPg/0JbhEeqUtBcllFwE3mdnXgDyC4wIvAt8xs9kE39r/Go7bAFwZ7mr670NdkLu/aGYvEOxOqiLYRdXhZuAPZlbr7m83s6sIrr9vwEPunu7a+8XA/WZWELb7wqHWJJKOTkkVEZFO2n0kIiKdtPtIhjwzu5HgvsupfuDut0RRj0g20+4jERHppN1HIiLSSaEgIiKdFAoiItJJoSAiIp3+P/KeIwHbROJ7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=2,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'binary:logistic',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb2_3, X_train, y_train, cv_folds = kfold)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
